# -*- coding: utf-8 -*-

import re
import os
import csv
import redis
import scrapy
import logging
import datetime
import pandas as pd
from copy import copy, deepcopy
from FDC_spider.constants import *
from scrapy.pipelines.images import ImagesPipeline
from FDC_spider.settings import today, IMAGES_STORE
from FDC_spider.utils.mongoclient import MongodbClient
from FDC_spider.utils.transfer_time import transfer_date
from FDC_spider.utils.case_id_dups import get_dup_case_id_li
from FDC_spider.utils.redis_pool import POOL, PROXY_REDIS_KEY
from FDC_spider.utils.proxy_to_redis_yueyang import PROXY_YY_KEY
from FDC_spider.utils.chq_dups import get_chq_dups_eg_li, get_chq_dups_bd_li
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class FdcSpiderPipeline(object):
    def process_item(self, item, spider):
        item['createTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['updateTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['fetchTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        return item


class RedisConnPipeline(object):
    def open_spider(self, spider):
        spider.redis_conn = redis.StrictRedis(connection_pool=POOL)
        spider.redis_conn.delete(PROXY_REDIS_KEY)

    def close_spider(self, spider):
        spider.redis_conn.close()


class MongoClientPipeline(object):
    def open_spider(self, spider):
        self.clo = MongodbClient()

    def close_spider(self, spider):
        self.clo.close()

    def process_item(self, item, spider):
        if spider.name == 'foshan':
            self.clo.set_db('zsq_fsfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'xiamen':
            # self.clo.set_db('zsqtest')
            if isinstance(item, FdcEstateGuidItem):
                # print('<estate>{}'.format(item))
                print('<estate>{}'.format(item.keys()))
                # self.clo.set_db('estate')
                # self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                # print('<building>{}'.format(item))
                print('<building>{}'.format(item.keys()))
                # self.clo.set_db('building')
                # self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                # print('<room>{}'.format(item))
                print('<room>{}'.format(item.keys()))
                # self.clo.set_db('room')
                # self.clo.insert_one(dict(item))
        if spider.name == 'beijing':
            self.clo.set_db('zsq_bjfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'kunming':
            self.clo.set_db('zsq_kmfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'wuxi':
            self.clo.set_db('zsq_wxfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'nanning':
            self.clo.set_db('zsq_nnfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'dongguan':
            self.clo.set_db('zsq_dgfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'shijiazhuang':
            self.clo.set_db('zsq_sjzfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'guiyang':
            self.clo.set_db('zsq_gyfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'zhongshan':
            self.clo.set_db('zsq_zsfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'huzhou':
            self.clo.set_db('zsq_hzfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'baoding':
            self.clo.set_db('zsq_bdfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'baoding_cert':
            self.clo.set_db('zsq_bdfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('permit')
                self.clo.insert_one(dict(item))
        if spider.name == 'changzhou':
            self.clo.set_db('zsq_czfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'nanjing':
            self.clo.set_db('zsq_njfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'shenzhen':
            self.clo.set_db('zsq_shzhfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'xuzhou':
            self.clo.set_db('zsq_xzhfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'sfpm':
            self.clo.set_db('sfpm')
            self.clo.set_col('case')
            self.clo.insert_one(dict(item))
        if spider.name == 'ganzhou':
            self.clo.set_db('zsq_gzhfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'yichang':
            self.clo.set_db('zsq_ychfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'xiangyang':
            self.clo.set_db('zsq_xyfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'wuhan':
            self.clo.set_db('zsq_whfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'nanchong':
            self.clo.set_db('zsq_nchfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'qingdao':
            self.clo.set_db('zsq_qdfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'zhuhai':
            self.clo.set_db('zsq_zhhfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'huanggang':
            self.clo.set_db('zsq_hgfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'chongqing':
            # self.clo.set_db('zsq_chqfc')
            self.clo.set_db('zsq_chqfc_2')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                # self.clo.insert_one(dict(item))
                filter_item = dict()
                filter_item['projectName'] = item['projectName']
                filter_item['projectId'] = item['projectId']
                filter_item['preSalePermit'] = item['preSalePermit']
                filter_item['preSaleBlockName'] = item['preSaleBlockName']
                self.clo.update_one(filter_item, dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                # self.clo.insert_one(dict(item))
                filter_item = dict()
                filter_item['projectId'] = item['projectId']
                filter_item['preSalePermit'] = item['preSalePermit']
                filter_item['blockName'] = item['blockName']
                filter_item['buildingId'] = item['buildingId']
                self.clo.update_one(filter_item, dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                # self.clo.insert_one(dict(item))
                filter_item = dict()
                filter_item['projectId'] = item['projectId']
                filter_item['preSalePermit'] = item['preSalePermit']
                filter_item['blockName'] = item['blockName']
                filter_item['buildingId'] = item['buildingId']
                filter_item['roomId'] = item['roomId']
                filter_item['roomNo'] = item['roomNo']
                filter_item['innerArea'] = item['innerArea']
                self.clo.update_one(filter_item, dict(item))
        if spider.name == 'lianyungang':
            self.clo.set_db('zsq_lygfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'yueyang':
            self.clo.set_db('zsq_yyfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'deyang':
            self.clo.set_db('zsq_dyfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'zhoushan':
            self.clo.set_db('zsq_zhshfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'guangan':
            self.clo.set_db('zsq_gafc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'ziyang':
            self.clo.set_db('zsq_zyfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'xiaogan':
            self.clo.set_db('zsq_xgfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'jingmen':
            self.clo.set_db('zsq_jmfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'tongren':
            self.clo.set_db('zsq_trfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'huangshi':
            self.clo.set_db('zsq_hshfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'shantou':
            self.clo.set_db('zsq_shtfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'jiujiang':
            self.clo.set_db('zsq_jjfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'luoyang':
            self.clo.set_db('zsq_lyfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'anqing':
            self.clo.set_db('zsq_aqfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        if spider.name == 'changchun':
            self.clo.set_db('zsq_ccfc')
            if isinstance(item, FdcEstateGuidItem):
                self.clo.set_col('estate')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcBuildingItem):
                self.clo.set_col('building')
                self.clo.insert_one(dict(item))
            if isinstance(item, FdcRoomItem):
                self.clo.set_col('room')
                self.clo.insert_one(dict(item))
        return item


# 佛山
class FoshanPipeline(object):
    """
    佛山市
    """

    def process_item(self, item, spider):
        if spider.name == 'foshan':
            item['city'] = '佛山市'
            item['sr'] = 'fsfc'
            item['siteName'] = '佛山市房产信息网'
        return item


class FoshanCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'foshan':
            self.estate_file = FoshanCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                  today.day, today.hour, today.minute,
                                                                  today.second)
            self.building_file = FoshanCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                    today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.room_file = FoshanCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (FS_EG_FLAG, FS_BD_FLAG, FS_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'foshan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 厦门
class XiamenPipeline(object):
    """
    厦门市
    """

    def process_item(self, item, spider):
        if spider.name == 'xiamen':
            item['city'] = '厦门市'
            item['sr'] = 'xmfc'
            item['siteName'] = '厦门网上房地产'
        return item


class XiamenCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'xiamen':
            self.estate_f = open(
                XiamenCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month, today.day,
                                                   today.hour,
                                                   today.minute, today.second), 'w',
                encoding='utf-8', newline='')
            self.building_f = open(
                XiamenCsvPipeline.file_temp.format(spider.name, 'building', today.year, today.month, today.day,
                                                   today.hour,
                                                   today.minute, today.second), 'w',
                encoding='utf-8', newline='')
            self.room_f = open(
                XiamenCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month, today.day, today.hour,
                                                   today.minute, today.second),
                'w',
                encoding='utf-8', newline='')
            self.estate_fw = csv.writer(self.estate_f)
            self.building_fw = csv.writer(self.building_f)
            self.room_fw = csv.writer(self.room_f)
            self.estate_fw.writerow(XM_EG_LI)
            self.building_fw.writerow(XM_BD_LI)
            self.room_fw.writerow(XM_RM_LI)

    def close_spider(self, spider):
        if spider.name == 'xiamen':
            self.estate_f.close()
            self.building_f.close()
            self.room_f.close()

    def process_item(self, item, spider):
        if spider.name == 'xiamen':
            if isinstance(item, FdcEstateGuidItem):
                self.estate_fw.writerow([item.get(i, None) for i in XM_EG_LI])
            if isinstance(item, FdcBuildingItem):
                self.building_fw.writerow([item.get(i, None) for i in XM_BD_LI])
            if isinstance(item, FdcRoomItem):
                self.room_fw.writerow([item.get(i, None) for i in XM_RM_LI])
        return item


# 北京
class BeijingPipeline(object):
    """
    北京市
    """

    def process_item(self, item, spider):
        if spider.name == 'beijing':
            item['city'] = '北京市'
            item['sr'] = 'bgfc'
            item['siteName'] = '北京市住房和城乡建设委员会'
            if isinstance(item, FdcEstateGuidItem):
                item.pop('estateTypeParams')
            if isinstance(item, FdcBuildingItem):
                pass
            if isinstance(item, FdcRoomItem):
                item.pop('priceDict')
        return item


class BeijingCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'beijing':
            self.estate_f = open(
                BeijingCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month, today.day,
                                                    today.hour,
                                                    today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.building_f = open(
                BeijingCsvPipeline.file_temp.format(spider.name, 'building', today.year, today.month, today.day,
                                                    today.hour,
                                                    today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.room_f = open(
                BeijingCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month, today.day, today.hour,
                                                    today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.estate_fw = csv.writer(self.estate_f)
            self.building_fw = csv.writer(self.building_f)
            self.room_fw = csv.writer(self.room_f)
            self.estate_fw.writerow(BJ_EG_LI)
            self.building_fw.writerow(BJ_BD_LI)
            self.room_fw.writerow(BJ_RM_LI)

    def close_spider(self, spider):
        if spider.name == 'beijing':
            self.estate_f.close()
            self.building_f.close()
            self.room_f.close()

    def process_item(self, item, spider):
        if spider.name == 'beijing':
            if isinstance(item, FdcEstateGuidItem):
                self.estate_fw.writerow([item.get(i, None) for i in BJ_EG_LI])
            if isinstance(item, FdcBuildingItem):
                self.building_fw.writerow([item.get(i, None) for i in BJ_BD_LI])
            if isinstance(item, FdcRoomItem):
                self.room_fw.writerow([item.get(i, None) for i in BJ_RM_LI])
        return item


# 昆明
class KunmingPipeline(object):
    """
    昆明市
    """

    def process_item(self, item, spider):
        if spider.name == 'kunming':
            item['city'] = '昆明市'
            item['sr'] = 'kmfc'
            item['siteName'] = '昆明市房产信息网'
        return item


class KunmingCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'kunming':
            self.estate_f = open(
                KunmingCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month, today.day,
                                                    today.hour,
                                                    today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.building_f = open(
                KunmingCsvPipeline.file_temp.format(spider.name, 'building', today.year, today.month, today.day,
                                                    today.hour,
                                                    today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.room_f = open(
                KunmingCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month, today.day, today.hour,
                                                    today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.estate_fw = csv.writer(self.estate_f)
            self.building_fw = csv.writer(self.building_f)
            self.room_fw = csv.writer(self.room_f)
            self.estate_fw.writerow(KM_EG_LI)
            self.building_fw.writerow(KM_BD_LI)
            self.room_fw.writerow(KM_RM_LI)

    def close_spider(self, spider):
        if spider.name == 'kunming':
            self.estate_f.close()
            self.building_f.close()
            self.room_f.close()

    def process_item(self, item, spider):
        if spider.name == 'kunming':
            if isinstance(item, FdcEstateGuidItem):
                self.estate_fw.writerow([item.get(i, None) for i in KM_EG_LI])
            if isinstance(item, FdcBuildingItem):
                self.building_fw.writerow([item.get(i, None) for i in KM_BD_LI])
            if isinstance(item, FdcRoomItem):
                self.room_fw.writerow([item.get(i, None) for i in KM_RM_LI])
        return item


# 无锡
class WuxiPipeline(object):
    """
    无锡市
    """

    def process_item(self, item, spider):
        if spider.name == 'wuxi':
            item['city'] = '无锡市'
            item['sr'] = 'wxfc'
            item['siteName'] = '无锡房地产市场网'
        return item


class WuxiCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'wuxi':
            self.estate_f = open(
                WuxiCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month, today.day,
                                                 today.hour,
                                                 today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.building_f = open(
                WuxiCsvPipeline.file_temp.format(spider.name, 'building', today.year, today.month, today.day,
                                                 today.hour,
                                                 today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.room_f = open(
                WuxiCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month, today.day, today.hour,
                                                 today.minute, today.second), 'w', encoding='utf-8', newline='')
            self.estate_fw = csv.writer(self.estate_f)
            self.building_fw = csv.writer(self.building_f)
            self.room_fw = csv.writer(self.room_f)
            self.estate_fw.writerow(WX_EG_LI)
            self.building_fw.writerow(WX_BD_LI)
            self.room_fw.writerow(WX_RM_LI)

    def close_spider(self, spider):
        if spider.name == 'wuxi':
            self.estate_f.close()
            self.building_f.close()
            self.room_f.close()

    def process_item(self, item, spider):
        if spider.name == 'wuxi':
            if isinstance(item, FdcEstateGuidItem):
                self.estate_fw.writerow([item.get(i, None) for i in WX_EG_LI])
            if isinstance(item, FdcBuildingItem):
                self.building_fw.writerow([item.get(i, None) for i in WX_BD_LI])
            if isinstance(item, FdcRoomItem):
                self.room_fw.writerow([item.get(i, None) for i in WX_RM_LI])
        return item


# 南宁
class NanningPipeline(object):
    """
    南宁市
    """

    def process_item(self, item, spider):
        if spider.name == 'nanning':
            item['city'] = '南宁市'
            item['sr'] = 'nnfc'
            item['siteName'] = '南宁选房网'
            if isinstance(item, FdcEstateGuidItem):
                landUsageTerm_li = re.findall(r'\d+', str(item['landUsageTerm'])) if item['landUsageTerm'] else []
                item['landUsageTerm'] = landUsageTerm_li[0] if len(landUsageTerm_li) else None
            if isinstance(item, FdcRoomItem):
                if item['unitName'] == '无':
                    item['unitName'] = None
                item.pop('unitNo')
        return item


class NanningCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'nanning':
            self.estate_file = NanningCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute, today.second)
            self.building_file = NanningCsvPipeline.file_temp.format(spider.name, 'building', today.year, today.month,
                                                                     today.day, today.hour, today.minute, today.second)
            self.room_file = NanningCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.NN_EG_FLAG, self.NN_BD_FLAG, self.NN_RM_FLAG = (NN_EG_FLAG, NN_BD_FLAG, NN_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'nanning':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.NN_EG_FLAG:
                    self.NN_EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.NN_BD_FLAG:
                    self.NN_BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.NN_RM_FLAG:
                    self.NN_RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 东莞
class DongguanPipeline(object):
    """
    东莞市
    """

    def process_item(self, item, spider):
        if spider.name == 'dongguan':
            item['city'] = '东莞市'
            item['sr'] = 'dgfc'
            item['siteName'] = '东莞市不动产登记中心'
        return item


class DongguanCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'dongguan':
            self.estate_file = DongguanCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                    today.day, today.hour, today.minute, today.second)
            self.building_file = DongguanCsvPipeline.file_temp.format(spider.name, 'building', today.year, today.month,
                                                                      today.day, today.hour, today.minute, today.second)
            self.room_file = DongguanCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                  today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (DG_EG_FLAG, DG_BD_FLAG, DG_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'dongguan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 石家庄
class ShijiazhuangPipeline(object):
    """
    石家庄
    """

    def process_item(self, item, spider):
        if spider.name == 'shijiazhuang':
            item['city'] = '石家庄市'
            item['sr'] = 'sjzfc'
            item['siteName'] = '石家庄市住房和城乡建设局'
        return item


class ShijiazhuangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'shijiazhuang':
            self.estate_file = ShijiazhuangCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                        today.day, today.hour, today.minute,
                                                                        today.second)
            self.building_file = ShijiazhuangCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                          today.month,
                                                                          today.day, today.hour, today.minute,
                                                                          today.second)
            self.room_file = ShijiazhuangCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                      today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (SJZ_EG_FLAG, SJZ_BD_FLAG, SJZ_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'shijiazhuang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 贵阳
class GuiyangPipeline(object):
    """
    贵阳市
    """

    def process_item(self, item, spider):
        if spider.name == 'guiyang':
            item['city'] = '贵阳市'
            item['sr'] = 'gyfc'
            item['siteName'] = '筑房网(贵阳住宅与房地产信息网)'
        return item


class GuiyangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'guiyang':
            self.estate_file = GuiyangCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.building_file = GuiyangCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                     today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.room_file = GuiyangCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (GY_EG_FLAG, GY_BD_FLAG, GY_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'guiyang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 中山
class ZhongshanPipeline(object):
    """
    中山市
    """

    def process_item(self, item, spider):
        if spider.name == 'zhongshan':
            item['city'] = '中山市'
            item['sr'] = 'zsfc'
            item['siteName'] = '中山市商品房网签备案情况查询系统'
        return item


class ZhongshanCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'zhongshan':
            self.estate_file = ZhongshanCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.building_file = ZhongshanCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                       today.month,
                                                                       today.day, today.hour, today.minute,
                                                                       today.second)
            self.room_file = ZhongshanCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (ZS_EG_FLAG, ZS_BD_FLAG, ZS_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'zhongshan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 湖州
class HuzhouPipeline(object):
    """
    湖州市
    """

    def process_item(self, item, spider):
        if spider.name == 'huzhou':
            item['city'] = '湖州市'
            item['sr'] = 'hzfc'
            item['siteName'] = '湖州市透明售房网'
        return item


class HuzhouCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'huzhou':
            self.estate_file = HuzhouCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                  today.day, today.hour, today.minute,
                                                                  today.second)
            self.building_file = HuzhouCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                    today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.room_file = HuzhouCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (HZ_EG_FLAG, HZ_BD_FLAG, HZ_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'huzhou':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 保定
class BaodingPipeline(object):
    """
    保定市
    """

    def process_item(self, item, spider):
        if spider.name == 'baoding' or spider.name == 'baoding_cert':
            item['city'] = '保定市'
            item['sr'] = 'bdfc'
            item['siteName'] = '保定市房地产市场管理处'
        return item


class BaodingCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'baoding':
            self.estate_file = BaodingCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.building_file = BaodingCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                     today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.room_file = BaodingCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (BD_EG_FLAG, BD_BD_FLAG, BD_RM_FLAG)
        if spider.name == 'baoding_cert':
            self.permit_file = BaodingCsvPipeline.file_temp.format(spider.name, 'permit', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.EG_FLAG = BD_EG_FLAG_P

    def process_item(self, item, spider):
        if spider.name == 'baoding':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        if spider.name == 'baoding_cert':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.permit_file, index=None, mode='a')
                else:
                    df.to_csv(self.permit_file, index=None, header=None, mode='a')
        return item


# 常州
class ChangzhouPipeline(object):
    """
    常州市
    """

    def process_item(self, item, spider):
        if spider.name == 'changzhou':
            item['city'] = '常州市'
            item['sr'] = 'czfc'
            item['siteName'] = '常州新建商品房信息公示'
        if isinstance(item, FdcEstateGuidItem):
            regex = re.compile(r'\d.*\d')
            if item['preSaleAccountNum']:
                ret_li = regex.findall(item['preSaleAccountNum'])
                item['preSaleAccountNum'] = ret_li[0] if len(ret_li) else None
        return item


class ChangzhouCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'changzhou':
            self.estate_file = ChangzhouCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.building_file = ChangzhouCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                       today.month,
                                                                       today.day, today.hour, today.minute,
                                                                       today.second)
            self.room_file = ChangzhouCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (CZ_EG_FLAG, CZ_BD_FLAG, CZ_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'changzhou':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 上海
class ShanghaiPipeline(object):
    """
    上海市
    """

    def process_item(self, item, spider):
        if spider.name == 'shanghai':
            item['city'] = '上海市'
            item['sr'] = 'shhfc'
            item['siteName'] = '网上房地产'
        return item


class ShanghaiCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'shanghai':
            self.estate_file = ShanghaiCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.building_file = ShanghaiCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                      today.month,
                                                                      today.day, today.hour, today.minute,
                                                                      today.second)
            self.room_file = ShanghaiCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                  today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (SHH_EG_FLAG, SHH_BD_FLAG, SHH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'shanghai':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 南京
class NanjingPipeline(object):
    """
    南京市
    """

    def process_item(self, item, spider):
        if spider.name == 'nanjing':
            item['city'] = '南京市'
            item['sr'] = 'njfc'
            item['siteName'] = '南京网上房地产'
        if isinstance(item, FdcEstateGuidItem):
            item['landUsageTerm'] = self.get_land_term(item['landUsageTerm'])
        return item

    @staticmethod
    def get_land_term(_str):
        regex_1 = re.compile(r'\xa0{1,}|，')
        regex_2 = re.compile(r'\d+.*\d+|\d+')
        if _str:
            ret_1 = regex_1.sub(',', _str).split(',')
            try:
                ret_2 = [regex_2.findall(i)[0] for i in ret_1]
            except:
                ret_2 = None
            return ret_2


class NanjingCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'nanjing':
            self.estate_file = NanjingCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.building_file = NanjingCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                     today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.room_file = NanjingCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (NJ_EG_FLAG, NJ_BD_FLAG, NJ_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'nanjing':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 深圳
class ShenzhenPipeline(object):
    """
    深圳市
    """

    def process_item(self, item, spider):
        if spider.name == 'shenzhen':
            item['city'] = '深圳市'
            item['sr'] = 'shzhfc'
            item['siteName'] = '深圳市房地产信息平台'
        return item


class ShenzhenCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'shenzhen':
            self.estate_file = ShenzhenCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.building_file = ShenzhenCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                      today.month,
                                                                      today.day, today.hour, today.minute,
                                                                      today.second)
            self.room_file = ShenzhenCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                  today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (SHZH_EG_FLAG, SHZH_BD_FLAG, SHZH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'shenzhen':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 徐州
class XuzhouPipeline(object):
    """
    徐州市
    """

    def process_item(self, item, spider):
        if spider.name == 'xuzhou':
            item['city'] = '徐州市'
            item['sr'] = 'xzhfc'
            item['siteName'] = '徐州市房地产信息网'
        return item


class XuzhouCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'xuzhou':
            self.estate_file = XuzhouCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                  today.day, today.hour, today.minute,
                                                                  today.second)
            self.building_file = XuzhouCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                    today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.room_file = XuzhouCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (XZH_EG_FLAG, XZH_BD_FLAG, XZH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'xuzhou':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 司法拍卖(阿里)
class SfpmCasePipeline(object):
    def open_spider(self, spider):
        if spider.name == 'sfpm':
            flag = spider.settings['SFPM_CASE_REDIS'] if spider.settings['SFPM_CASE_REDIS'] != None else True
            if flag:
                logger.info('>>>>开始去重<<<<')
                ret = get_dup_case_id_li()
                logger.info('>>>>去重完成<<<<') if ret else logger.error('>>>>去重失败<<<<')

    def process_item(self, item, spider):
        item['updateTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['fetchTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        return item


class SfpmCaseImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        headers = dict(referer=item['caseUrl'])
        for img_url in item['imageUrls']:
            yield scrapy.Request(
                img_url,
                headers=headers,
                meta=dict(item=deepcopy(item)),
            )

    # 修改文件的命名和路径
    def file_path(self, request, response=None, info=None):
        item = copy(request.meta['item'])
        image_guid = request.url.split('/')[-1]
        file_path = '{}/{}/{}/{}/{}'.format(item['province'], item['city'], item['districtName'],
                                            item['categoryName'], item['caseId'], )
        full_path = os.path.join(IMAGES_STORE, file_path)
        if not os.path.exists(full_path):
            os.makedirs(full_path)
        file_name = os.path.join(file_path, image_guid)
        return file_name


# 赣州
class GanzhouPipeline(object):
    """
    赣州市
    """

    def process_item(self, item, spider):
        if spider.name == 'ganzhou':
            item['city'] = '赣州市'
            item['sr'] = 'gzhfc'
            item['siteName'] = '赣州市赣州房信网'
            if isinstance(item, FdcRoomItem):
                item['roomStructure'] = None if item['roomStructure'] == '---' else item['roomStructure']
        return item


class GanzhouCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'ganzhou':
            self.estate_file = GanzhouCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.building_file = GanzhouCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                     today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.room_file = GanzhouCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (GZH_EG_FLAG, GZH_BD_FLAG, GZH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'ganzhou':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 宜昌
class YichangPipeline(object):
    """
    宜昌市
    """

    def process_item(self, item, spider):
        if spider.name == 'yichang':
            item['city'] = '宜昌市'
            item['sr'] = 'ychfc'
            item['siteName'] = '宜昌市不动产交易登记'
        return item


class YichangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'yichang':
            self.estate_file = YichangCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.building_file = YichangCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                     today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.room_file = YichangCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (YCH_EG_FLAG, YCH_BD_FLAG, YCH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'yichang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 襄阳
class XiangyangPipeline(object):
    """
    襄阳市
    """

    def process_item(self, item, spider):
        if spider.name == 'xiangyang':
            if isinstance(item, FdcEstateGuidItem):
                # 楼盘字段清洗
                item['buildingCategory'] = self.deal_building_category(item['buildingCategory'])
                item['certDate'] = self.deal_cert_date(item['certDate'])
                item['permitSalesArea'] = self.deal_permit_sales_area(item['permitSalesArea'])
                item['permitBuildingNum'] = self.deal_permit_building_num(item['permitBuildingNum'])
            if isinstance(item, FdcRoomItem):
                # 房号字段清洗
                item['roomStructure'] = self.deal_room_structure(item['roomStructure'])
                item['undecoratedPrice'] = self.deal_undecorated_price(item['undecoratedPrice'])
            item['city'] = '襄阳市'
            item['sr'] = 'xyfc'
            item['siteName'] = '襄阳市住房和城乡建设局'
        return item

    @staticmethod
    def deal_building_category(_str):
        """
        建筑类别 字段清洗
        :param _str: <str> or None
        :return: <str> or None
        """
        regex = re.compile(r'\s', re.S)
        try:
            ret = regex.sub('', _str)
            ret = ret.split(',') if ret else None
        except:
            return
        else:
            return ret

    @staticmethod
    def deal_cert_date(_str):
        """
        发证日期 字段清洗
        :param _str: <str> or None
        :return: <str> or None
        """
        try:
            ret = _str.split('：')[1]
            assert len(ret) > 4
        except:
            return
        else:
            return ret

    @staticmethod
    def deal_permit_sales_area(_str):
        """
        许可证销售范围 字段清洗
        :param _str: <str> or None
        :return: <str> or None
        """
        try:
            ret = _str.split('：')[1].strip()
            assert len(ret) > 3
        except:
            return
        else:
            return ret

    @staticmethod
    def deal_permit_building_num(_str):
        """
        许可证楼栋号 字段清洗
        :param _str: <str> or None
        :return: <str> or None
        """
        try:
            ret = _str.split('：')[1].split(',')
            assert ret
            ret = [i.strip() for i in ret if i.strip()]
        except:
            return
        else:
            return ret

    @staticmethod
    def deal_room_structure(_str):
        """
        房号户型 字段清洗
        :param _str: <str> or None
        :return: <str> or None
        """
        regex = re.compile(r'\d+')
        try:
            ret = regex.findall(_str)
            assert [i for i in ret if int(i)]
        except:
            return
        else:
            return _str

    @staticmethod
    def deal_undecorated_price(_str):
        """
        挂牌清水价 字段清洗
        :param _str: <str> or None
        :return: <str> or None
        """
        try:
            ret = float(_str)
            assert ret
        except:
            return
        else:
            return _str


class XiangyangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'xiangyang':
            self.estate_file = XiangyangCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.building_file = XiangyangCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                       today.month,
                                                                       today.day, today.hour, today.minute,
                                                                       today.second)
            self.room_file = XiangyangCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (XY_EG_FLAG, XY_BD_FLAG, XY_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'xiangyang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 武汉
class WuhanPipeline(object):
    """
    武汉市
    """

    def process_item(self, item, spider):
        if spider.name == 'wuhan':
            if isinstance(item, FdcEstateGuidItem):
                key_li = list()
                # 对楼盘字典进行初步清洗，空值改为None
                for item_tup in item.items():
                    try:
                        if '/' in item_tup[1] and item_tup[1].find('/') < 2:
                            key_li.append(item_tup[0])
                    except:
                        continue
                if key_li:
                    for key_str in key_li:
                        item[key_str] = None
                # 对楼盘许可证进行初步清洗，去掉换行符和空格
                if item['preSalePermit']:
                    item['preSalePermit'] = [str(i).strip() for i in item['preSalePermit'] if i]
            if isinstance(item, FdcRoomItem):
                rm_key_li = list()
                # 对房号字典进行初步清洗，空值改为None
                for item_tup in item.items():
                    try:
                        if '/' in item_tup[1] and item_tup[1].find('/') < 2:
                            rm_key_li.append(item_tup[0])
                    except:
                        continue
                if rm_key_li:
                    for key_str in rm_key_li:
                        item[key_str] = None
            item['city'] = '武汉市'
            item['sr'] = 'whfc'
            item['siteName'] = '武汉市住房保障和房屋管理局'
        return item


class WuhanCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'wuhan':
            self.estate_file = WuhanCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                 today.day, today.hour, today.minute,
                                                                 today.second)
            self.building_file = WuhanCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                   today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.room_file = WuhanCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                               today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (WH_EG_FLAG, WH_BD_FLAG, WH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'wuhan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 南充
class NanchongPipeline(object):
    """
    南充市
    """

    def process_item(self, item, spider):
        if spider.name == 'nanchong':
            item['city'] = '南充市'
            item['sr'] = 'nchfc'
            item['siteName'] = '南充市房地产管理局'
        return item


class NanchongCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'nanchong':
            self.estate_file = NanchongCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.building_file = NanchongCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                      today.month,
                                                                      today.day, today.hour, today.minute,
                                                                      today.second)
            self.room_file = NanchongCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                  today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (NCHO_EG_FLAG, NCHO_BD_FLAG, NCHO_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'nanchong':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 青岛
class QingdaoPipeline(object):
    """
    青岛市
    """

    def process_item(self, item, spider):
        if spider.name == 'qingdao':
            if isinstance(item, FdcEstateGuidItem):
                # 对楼盘字典进行初步清洗，空值改为None
                for item_eg_tup in item.items():
                    if self._deal_item_eg(item_eg_tup[1]):
                        item[item_eg_tup[0]] = None
            if isinstance(item, FdcRoomItem):
                # 对房号字段进行初步清洗
                for item_rm_tup in item.items():
                    item[item_rm_tup[0]] = self._deal_item_rm(item_rm_tup[1])
            item['city'] = '青岛市'
            item['sr'] = 'qdfc'
            item['siteName'] = '青岛市网上房地产'
        return item

    @staticmethod
    def _deal_item_eg(_str):
        regex = re.compile(r'\d+')
        try:
            ret_li = regex.findall(_str)
            ret = int(ret_li[0])
            assert len(ret_li) < 2 and ret == 0
        except:
            return False
        else:
            return True

    @staticmethod
    def _deal_item_rm(_str):
        try:
            ret = _str.strip()
        except:
            return _str
        else:
            return ret


class QingdaoCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'qingdao':
            self.estate_file = QingdaoCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                   today.day, today.hour, today.minute,
                                                                   today.second)
            self.building_file = QingdaoCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                     today.month,
                                                                     today.day, today.hour, today.minute,
                                                                     today.second)
            self.room_file = QingdaoCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                 today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (QD_EG_FLAG, QD_BD_FLAG, QD_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'qingdao':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 珠海
class ZhuhaiPipeline(object):
    """
    珠海市
    """

    def process_item(self, item, spider):
        if spider.name == 'zhuhai':
            if isinstance(item, FdcRoomItem):
                # 对房号字段进行初步清洗
                for item_rm_tup in item.items():
                    item[item_rm_tup[0]] = self._deal_item_rm(item_rm_tup[1])
            item['city'] = '珠海市'
            item['sr'] = 'zhhfc'
            item['siteName'] = '珠海市商品房预（销）售专网'
        return item

    @staticmethod
    def _deal_item_rm(_str):
        try:
            assert _str and _str != '/'
        except:
            return
        else:
            return _str


class ZhuhaiCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'zhuhai':
            self.estate_file = ZhuhaiCsvPipeline.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                                  today.day, today.hour, today.minute,
                                                                  today.second)
            self.building_file = ZhuhaiCsvPipeline.file_temp.format(spider.name, 'building', today.year,
                                                                    today.month,
                                                                    today.day, today.hour, today.minute,
                                                                    today.second)
            self.room_file = ZhuhaiCsvPipeline.file_temp.format(spider.name, 'room', today.year, today.month,
                                                                today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (ZHH_EG_FLAG, ZHH_BD_FLAG, ZHH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'zhuhai':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 黄冈
class HuanggangPipeline(object):
    """
    黄冈市
    """

    def process_item(self, item, spider):
        if spider.name == 'huanggang':
            item['city'] = '黄冈市'
            item['sr'] = 'hgfc'
            item['siteName'] = '黄冈市房地产市场信息网'
        return item


class HuanggangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'huanggang':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (HG_EG_FLAG, HG_BD_FLAG, HG_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'huanggang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 重庆
class ChongqingPipeline(object):
    """
    重庆市
    """

    def open_spider(self, spider):
        if spider.name == 'chongqing':
            chq_dups_eg = spider.settings['CHQ_DUPS_EG'] if spider.settings['CHQ_DUPS_EG'] != None else True
            chq_dups_bd = spider.settings['CHQ_DUPS_BD'] if spider.settings['CHQ_DUPS_BD'] != None else True
            if chq_dups_eg:
                logger.info(f'>>>>{spider.name}:项目去重开启<<<<')
                ret_eg = get_chq_dups_eg_li()
                logger.info(f'>>>>{spider.name}:项目去重配置成功<<<<') if ret_eg else logger.error(
                    f'>>>>{spider.name}:项目去重配置失败<<<<')
            if chq_dups_bd:
                logger.info(f'>>>>{spider.name}:楼栋去重开启<<<<')
                ret_bd = get_chq_dups_bd_li()
                logger.info(f'>>>>{spider.name}:楼栋去重配置完成<<<<') if ret_bd else logger.error(
                    f'>>>>{spider.name}:楼栋去重配置失败<<<<')

    def process_item(self, item, spider):
        if spider.name == 'chongqing':
            if isinstance(item, FdcRoomItem):
                # 对房号用途字段进行初步清洗
                item['roomUse'] = self._deal_room_use(item['roomUse'])
            item['city'] = '重庆市'
            item['sr'] = 'chqfc'
            item['siteName'] = '重庆网上房地产'
        return item

    @staticmethod
    def _deal_room_use(_str):
        try:
            ret = _str.strip() if isinstance(_str, str) else _str
            assert ret
        except:
            return '其他'
        else:
            return _str


class ChongqingCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'chongqing':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (CHQ_EG_FLAG, CHQ_BD_FLAG, CHQ_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'chongqing':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 连云港
class LianyungangPipeline(object):
    """
    连云港市
    """

    def process_item(self, item, spider):
        if spider.name == 'lianyungang':
            if isinstance(item, FdcEstateGuidItem):
                # 对楼盘字段进行初步清洗
                for item_eg_tup in item.items():
                    item[item_eg_tup[0]] = self._deal_item_eg(item_eg_tup[1])
            if isinstance(item, FdcRoomItem):
                # 对房号字段进行初步清洗
                for item_rm_tup in item.items():
                    item[item_rm_tup[0]] = self._deal_item_rm(item_rm_tup[1])
                # 清洗价格字段
                item['referencePrice'] = self._deal_item_rm_price(item['referencePrice'])
            item['city'] = '连云港市'
            item['sr'] = 'lygfc'
            item['siteName'] = '连云港房地产网'
        return item

    @staticmethod
    def _deal_item_eg(_str):
        try:
            if isinstance(_str, str):
                ret = _str.strip()
            else:
                ret = [i.strip() for i in _str if i.strip()]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_item_rm(_str):
        try:
            assert _str and '无 ' not in _str and '_' not in _str and '未知' not in _str
        except:
            return
        else:
            return _str

    @staticmethod
    def _deal_item_rm_price(_str):
        regex = re.compile(r'\d+')
        try:
            ret = regex.findall(_str)[0]
            assert ret
        except:
            return
        else:
            return ret


class LianyungangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'lianyungang':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (LYG_EG_FLAG, LYG_BD_FLAG, LYG_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'lianyungang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 岳阳市
class YueyangPipeline(object):
    """
    岳阳市
    """

    def process_item(self, item, spider):
        if spider.name == 'yueyang':
            item['city'] = '岳阳市'
            item['sr'] = 'yyfc'
            item['siteName'] = '岳阳房地产网'
        return item


class YueyangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'yueyang':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (YY_EG_FLAG, YY_BD_FLAG, YY_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'yueyang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


class YueyangRedisConnPipeline(object):
    def open_spider(self, spider):
        if spider.name == 'yueyang':
            spider.yy_redis_conn = redis.StrictRedis(connection_pool=POOL)
            spider.yy_redis_conn.delete(PROXY_YY_KEY)

    def close_spider(self, spider):
        if spider.name == 'yueyang':
            spider.yy_redis_conn.close()


# 德阳市
class DeyangPipeline(object):
    """
    德阳市
    """

    def process_item(self, item, spider):
        if spider.name == 'deyang':
            item['city'] = '德阳市'
            item['sr'] = 'dyfc'
            item['siteName'] = '德阳房地产'
        return item


class DeyangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'deyang':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (DY_EG_FLAG, DY_BD_FLAG, DY_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'deyang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 舟山市
class ZhoushanPipeline(object):
    """
    舟山市
    """

    def process_item(self, item, spider):
        if spider.name == 'zhoushan':
            item['city'] = '舟山市'
            item['sr'] = 'zhshfc'
            item['siteName'] = '舟山透明售房网'
        return item


class ZhoushanCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'zhoushan':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (ZHSH_EG_FLAG, ZHSH_BD_FLAG, ZHSH_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'zhoushan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 广安市
class GuanganPipeline(object):
    """
    广安市
    """

    def process_item(self, item, spider):
        if spider.name == 'guangan':
            item['city'] = '广安市'
            item['sr'] = 'gafc'
            item['siteName'] = '广安市房地产交易服务平台'
            # 对楼盘字段进行初步清洗
            if isinstance(item, FdcEstateGuidItem):
                for item_eg_tup in item.items():
                    item[item_eg_tup[0]] = self._deal_item_eg(item_eg_tup[1])
        return item

    @staticmethod
    def _deal_item_eg(_ret):
        try:
            if isinstance(_ret, str):
                ret = _ret.strip()
            elif isinstance(_ret, list):
                ret = [str(i).strip() if i else None for i in _ret]
            else:
                ret = _ret
            assert ret
        except:
            return
        else:
            return ret


class GuanganCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'guangan':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (GUANGAN_EG_FLAG, GUANGAN_BD_FLAG, GUANGAN_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'guangan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 资阳市
class ZiyangPipeline(object):
    """
    资阳市
    """

    def process_item(self, item, spider):
        if spider.name == 'ziyang':
            item['city'] = '资阳市'
            item['sr'] = 'zyfc'
            item['siteName'] = '资阳市房地产管理局'
            # 对楼盘字段进行初步清洗
            if isinstance(item, FdcEstateGuidItem):
                for item_eg_tup in item.items():
                    item[item_eg_tup[0]] = self._deal_item_eg(item_eg_tup[1])
        return item

    @staticmethod
    def _deal_item_eg(_ret):
        try:
            if isinstance(_ret, str):
                ret = _ret.strip()
            elif isinstance(_ret, list):
                ret = [str(i).strip() if i else None for i in _ret]
            else:
                ret = _ret
            assert ret
        except:
            return
        else:
            return ret


class ZiyangCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'ziyang':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (ZIYANG_EG_FLAG, ZIYANG_BD_FLAG, ZIYANG_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'ziyang':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


#  孝感市
class XiaoganPipeline(object):
    """
    孝感市
    """

    def process_item(self, item, spider):
        if spider.name == 'xiaogan':
            item['city'] = '孝感市'
            item['sr'] = 'xgfc'
            item['siteName'] = '房地产市场信息发布平台'
            # 对房号字段进行初步清洗
            if isinstance(item, FdcRoomItem):
                for item_rm_tup in item.items():
                    item[item_rm_tup[0]] = self._deal_item_rm(item_rm_tup[1])
        return item

    @staticmethod
    def _deal_item_rm(_ret):
        try:
            if isinstance(_ret, str):
                ret = _ret.strip()
            elif isinstance(_ret, list):
                ret = [str(i).strip() if i else None for i in _ret]
            else:
                ret = _ret
            assert ret
        except:
            return
        else:
            return ret


class XiaoganCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'xiaogan':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (XIAOGAN_EG_FLAG, XY_BD_FLAG, XY_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'xiaogan':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


#  荆门市
class JingmenPipeline(object):
    """
    荆门市
    """

    def process_item(self, item, spider):
        if spider.name == 'jingmen':
            item['city'] = '荆门市'
            item['sr'] = 'jmfc'
            item['siteName'] = '荆门房产网'
            # 对楼盘字段进行初步清洗
            if isinstance(item, FdcEstateGuidItem):
                for item_eg_tup in item.items():
                    item[item_eg_tup[0]] = self._deal_item_eg(item_eg_tup[1])
            # 对房号字段进行初步清洗
            if isinstance(item, FdcRoomItem):
                for item_rm_tup in item.items():
                    item[item_rm_tup[0]] = self._deal_item_rm(item_rm_tup[1])
        return item

    @staticmethod
    def _deal_item_eg(_ret):
        try:
            if isinstance(_ret, str):
                ret = _ret.strip()
            elif isinstance(_ret, list):
                ret = [str(i).strip() if i else None for i in _ret]
            else:
                ret = _ret
            assert ret
            assert ('待定' and '不详' and '待更新') not in ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_item_rm(_ret):
        try:
            if isinstance(_ret, str):
                ret = _ret.strip()
                assert '暂无' not in ret
            else:
                ret = _ret
        except:
            return
        else:
            return ret


class JingmenCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'jingmen':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (JINGMEN_EG_FLAG, JINGMEN_BD_FLAG, JINGMEN_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'jingmen':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


#  铜仁市
class TongrenPipeline(object):
    """
    铜仁市
    """

    def process_item(self, item, spider):
        if spider.name == 'tongren':
            item['city'] = '铜仁市'
            item['sr'] = 'trfc'
            item['siteName'] = '铜仁市房地产交易管理处'
            # 对房号字段进行初步清洗
            if isinstance(item, FdcRoomItem):
                # 建成时间
                item['buildDate'] = self.stamp_to_date(item['buildDate'])
                # 是否跃层
                item['isDuplex'] = self.str_to_bool(item['isDuplex'])
                # 是否电梯
                item['isElevator'] = self.str_to_bool(item['isElevator'])
        return item

    @staticmethod
    def stamp_to_date(stamp: str) -> str or None:
        regex = re.compile(r'\d+')
        try:
            stamp_li: list = regex.findall(stamp)
            assert len(stamp_li) == 1
            stamp_int: int = int(stamp_li[0])
            date_str: str = transfer_date(stamp_int)
        except:
            return
        else:
            return date_str

    @staticmethod
    def str_to_bool(_str: str) -> bool:
        return True if _str == '1' or _str == 1 else False


class TongrenCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'tongren':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (TONGREN_EG_FLAG, TONGREN_BD_FLAG, TONGREN_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'tongren':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


#  黄石市
class HuangshiPipeline(object):
    """
    黄石市
    """

    def process_item(self, item, spider):
        if spider.name == 'huangshi':
            item['city'] = '黄石市'
            item['sr'] = 'hshfc'
            item['siteName'] = '黄石市房地产市场信息发布平台'
            # 对楼盘/楼栋字段进行初步清洗
            if not isinstance(item, FdcRoomItem):
                for item_eg_tup in item.items():
                    item[item_eg_tup[0]] = self._deal_item_space(item_eg_tup[1])
            # 对房号销售状态字段进行初步清洗
            else:
                item['saleStatus'] = self._deal_sale_status(item['saleStatus'])
        return item

    @staticmethod
    def _deal_item_space(_ret):
        regex = re.compile(r'\s{2,}', re.S)
        if isinstance(_ret, str):
            try:
                assert _ret
                ret = regex.sub('', _ret)
                assert ret
            except:
                return
            else:
                return ret
        else:
            return _ret

    @staticmethod
    def _deal_sale_status(_li: list) -> str or None:
        """
        获取销售状态
        :param _li:
        :return:
        """
        try:
            ret = ''.join([str(i).strip() for i in _li if i and str(i).strip()])
            assert ret
        except:
            return
        else:
            return ret


class HuangshiCsvPipeline(object):
    file_temp = 'FDC_spider/csv_file/{}_{}_{}_{}_{}_{}{}{}.csv'

    def open_spider(self, spider):
        if spider.name == 'huangshi':
            self.estate_file = self.file_temp.format(spider.name, 'estate', today.year, today.month,
                                                     today.day, today.hour, today.minute,
                                                     today.second)
            self.building_file = self.file_temp.format(spider.name, 'building', today.year,
                                                       today.month,
                                                       today.day, today.hour, today.minute,
                                                       today.second)
            self.room_file = self.file_temp.format(spider.name, 'room', today.year, today.month,
                                                   today.day, today.hour, today.minute, today.second)
            self.EG_FLAG, self.BD_FLAG, self.RM_FLAG = (HUANGSHI_EG_FLAG, HUANGSHI_BD_FLAG, HUANGSHI_RM_FLAG)

    def process_item(self, item, spider):
        if spider.name == 'huangshi':
            item_li = [dict(item)]
            df = pd.DataFrame(item_li)
            if isinstance(item, FdcEstateGuidItem):
                if self.EG_FLAG:
                    self.EG_FLAG = False
                    df.to_csv(self.estate_file, index=None, mode='a')
                else:
                    df.to_csv(self.estate_file, index=None, header=None, mode='a')
            if isinstance(item, FdcBuildingItem):
                if self.BD_FLAG:
                    self.BD_FLAG = False
                    df.to_csv(self.building_file, index=None, mode='a')
                else:
                    df.to_csv(self.building_file, index=None, header=None, mode='a')
            if isinstance(item, FdcRoomItem):
                if self.RM_FLAG:
                    self.RM_FLAG = False
                    df.to_csv(self.room_file, index=None, mode='a')
                else:
                    df.to_csv(self.room_file, index=None, header=None, mode='a')
        return item


# 汕头市
class ShantouPipeline(object):
    """
    汕头市
    """

    def process_item(self, item, spider):
        if spider.name == 'shantou':
            item['city'] = '汕头市'
            item['sr'] = 'shtfc'
            item['siteName'] = '汕头市商品房管理信息平台'
            # 对楼盘/楼栋字段进行初步清洗
            for item_eg_tup in item.items():
                item[item_eg_tup[0]] = self._deal_item_space(item_eg_tup[1])
            # 对房号部分字段进行初步清洗
            if isinstance(item, FdcRoomItem):
                item['saleStatus'] = self._deal_sale_status(item['saleStatus'])
                item['recordTotalPrice'] = self._deal_total_price(item['recordTotalPrice'])
                item['nominalFloor'] = self._deal_floor_num(item['nominalFloor'])
                item['physicsFloor'] = self._deal_floor_num(item['physicsFloor'])
        return item

    @staticmethod
    def _deal_item_space(_ret):
        """
        处理空字符
        :param _ret:
        :return:
        """
        regex = re.compile(r'\s{2,}', re.S)
        if isinstance(_ret, str):
            try:
                assert _ret.strip()
                ret = regex.sub('', _ret)
                assert ret
            except:
                return
            else:
                return ret
        else:
            return _ret

    @staticmethod
    def _deal_sale_status(_li: list) -> str or None:
        """
        处理销售状态
        :param _li:
        :return:
        """
        try:
            ret = ''.join([str(i).strip() for i in _li if i and str(i).strip()])
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_total_price(_str: str):
        """
        处理备案总价
        :param _str:
        :return:
        """
        try:
            ret = int(_str)
            if ret > 10 ** 5:
                return ret
            else:
                return f'{ret}万元'
        except:
            return

    @staticmethod
    def _deal_floor_num(_cont) -> int or None:
        """
        处理楼层
        :param _cont:
        :return:
        """
        try:
            ret = int(_cont)
        except:
            return
        else:
            return ret


# 九江市
class JiujiangPipeline(object):
    """
    九江市
    """

    def process_item(self, item, spider):
        if spider.name == 'jiujiang':
            item['city'] = '九江市'
            item['sr'] = 'jjfc'
            item['siteName'] = '九江市房地产信息网'
            # 对房号部分字段进行初步清洗
            if isinstance(item, FdcRoomItem):
                item['saleStatus'] = self._deal_sale_status(item['saleStatus'])
            # 处理空白字段
            for item_eg_tup in item.items():
                item[item_eg_tup[0]] = self._deal_item_space(item_eg_tup[1])
        return item

    @staticmethod
    def _deal_item_space(_ret):
        """
        处理空字符
        :param _ret:
        :return:
        """
        regex = re.compile(r'\s{2,}', re.S)
        if isinstance(_ret, str):
            try:
                assert _ret.strip()
                ret = regex.sub('', _ret)
                assert ret
            except:
                return
            else:
                return ret
        else:
            return _ret

    @staticmethod
    def _deal_sale_status(_cont: str):
        """
        处理销售状态
        :param _li:
        :return:
        """
        try:
            assert isinstance(_cont, str)
            if not _cont.strip():
                ret = '不可售'
            elif '最高额' in _cont:
                ret = '抵押'
            else:
                ret = _cont
        except:
            return
        else:
            return ret


# 洛阳市
class LuoyangPipeline(object):
    """
    洛阳市
    """

    def process_item(self, item, spider):
        if spider.name == 'luoyang':
            item['city'] = '洛阳市'
            item['sr'] = 'lyfc'
            item['siteName'] = '洛阳市住房保障和房产服务中心'
        return item


# 安庆市
class AnqingPipeline(object):
    """
    安庆市
    """

    def process_item(self, item, spider):
        if spider.name == 'anqing':
            item['city'] = '安庆市'
            item['sr'] = 'aqfc'
            item['siteName'] = '商品房销售许可证查询系统'
        return item


# 长春市
class ChangchunPipeline(object):
    """
    长春市
    """

    def process_item(self, item, spider):
        if spider.name == 'changchun':
            item['city'] = '长春市'
            item['sr'] = 'ccfc'
            item['siteName'] = '长春市房产信息平台'
            # str空值处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item_str(item_tup[1])
        return item

    @staticmethod
    def deal_item_str(item_val):
        """
        案例字段str空值处理
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert isinstance(item_val, str)
            item_val = item_val.strip() if item_val.strip() else None
        except:
            return item_val
        else:
            return item_val
